import trio
import asks
import logging
import json
import time

from urllib.parse import quote
from termcolor import colored
from pyquery import PyQuery as jq

logging.basicConfig(level=logging.INFO,
                    format='[%(asctime)s] >>> %(levelname)s  %(name)s: %(message)s')
Loger = logging.getLogger('demo')
asks.init('trio')

keyword = quote("霸座") # 初始化要爬的关键字
session = asks.Session(headers={
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive",
    "Cookie": "BAIDUID=4DFBAB65A55BB817DA82151156969593:FG=1; BIDUPSID=4DFBAB65A55BB817DA82151156969593; PSTM=1531210011; TIEBA_USERTYPE=ab202a74b974084564dafb3a; bdshare_firstime=1533367063193; IS_NEW_USER=cee2d6fee19ba4436ed71bfe; TIEBAUID=ff410537a1ef8ddca1baa073; BDUSS=dTYi1qUm80b1NDTDIxWFRzVnptam1ZblBxM3dLNENIeGdveGUwZnNLQXBOTXhiQVFBQUFBJCQAAAAAAAAAAAEAAABxe5wBYTI5MzEwMjQ2AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACmnpFspp6Rbc2; STOKEN=6621f14fecc0224e1b6b0dd1223fbd3736398d7ce993dab8b4ce03012d07b114; H_PS_PSSID=1459_21084_22160; Hm_lvt_98b9d8c2fd6608d564bf2ac2ae642948=1538013024,1538014279,1538029615,1538116308; 27032433_FRSVideoUploadTip=1; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; delPer=0; PSINO=5; Hm_lpvt_98b9d8c2fd6608d564bf2ac2ae642948=1538116501",
    "Host": "tieba.baidu.com",
    "Referer": "http://tieba.baidu.com/f/search/res?ie=utf-8&qw=%E9%9C%B8%E5%BA%A7",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",
}, connections=10) #初始化一个session，设置并发连接5个

alllist = []

async def tiebacontent(title,tid,content,time):
    mylist = []
    bean = {"title":title,"tid":tid,"content":content,"time":time,"mylist":mylist}
    alllist.append(bean)
    url = f"https://tieba.baidu.com/p/{tid}"
    Loger.info(colored(f'请求: {url}', 'yellow'))  # 打个log发送请求 颜色黄色
    resp = await asks.get(url)
    num = jq(resp.text)('#thread_theme_7 > div.l_thread_info > ul > li:nth-child(2) > span:nth-child(2)').text()
    if num:
        for i in range(1,int(num)+1):
            urls = url + f"?pn={i}"
            resp = await  asks.get(urls)
            for content in jq(resp.text)('#j_p_postlist > div').items():
                aaa = content('.d_post_content_main >.p_content > cc > div').text()
                name = content('div.d_author > ul > li.d_name > a').text()
                mylist.append({"name":name,"content":aaa})

async def tieba(page):
    Loger.info(colored(f'请求: {page}', 'yellow'))  # 打个log发送请求 颜色黄色
    url = f"http://tieba.baidu.com/f/search/res?ie=utf-8&qw={keyword}&rn=10&only_thread=1&pn={page}"
    Loger.info(colored(f'请求: {url}','yellow'))  # 打个log发送请求 颜色黄色
    resp = await asks.get(url)
    aaa = jq(resp.text)('.s_post_list')
    for i in aaa('.s_post').items():
        title = i('.p_title').text()
        tid = i('.p_title > a').attr('data-tid')
        content = i('.p_content').text()
        time = i('div > font').text()
        await tiebacontent(title,tid,content,time)



async def main():
    async with trio.open_nursery() as nursery:
        for page in range(1,100):
            nursery.start_soon(tieba,page)


def save():
    with open('tieba.json','w',encoding='utf-8') as f:
        f.write(json.dumps(alllist,indent=4,ensure_ascii=False))


if __name__ == '__main__':
    # trio.run(main)
    StartTime = time.time()
    try:
        trio.run(main)  # 启动main 直到结束
    except KeyboardInterrupt:  # 如果是 ctrl+c 导致结束 那不显示任何信息
        pass
    except Exception as e:  # 如果是其他错误，那打个log，颜色红色
        Loger.error(colored(e, 'red'))
    finally:
        Loger.info(colored(f"消耗时间: {int(time.time()-StartTime)}秒"))
        save()
